{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup IAM for Kinesis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import pandas as pd\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "bucket = sess.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "region = boto3.Session().region_name\n",
    "\n",
    "sts = boto3.Session().client(service_name=\"sts\", region_name=region)\n",
    "iam = boto3.Session().client(service_name=\"iam\", region_name=region)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Kinesis Role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_kinesis_role_name = \"DSOAWS_Kinesis\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_kinesis_role_passed = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assume_role_policy_doc = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\"Effect\": \"Allow\", \"Principal\": {\"Service\": \"kinesis.amazonaws.com\"}, \"Action\": \"sts:AssumeRole\"},\n",
    "        {\"Effect\": \"Allow\", \"Principal\": {\"Service\": \"firehose.amazonaws.com\"}, \"Action\": \"sts:AssumeRole\"},\n",
    "        {\"Effect\": \"Allow\", \"Principal\": {\"Service\": \"kinesisanalytics.amazonaws.com\"}, \"Action\": \"sts:AssumeRole\"},\n",
    "    ],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "try:\n",
    "    iam_role_kinesis = iam.create_role(\n",
    "        RoleName=iam_kinesis_role_name,\n",
    "        AssumeRolePolicyDocument=json.dumps(assume_role_policy_doc),\n",
    "        Description=\"DSOAWS Kinesis Role\",\n",
    "    )\n",
    "    print(\"Role succesfully created.\")\n",
    "    iam_kinesis_role_passed = True\n",
    "except ClientError as e:\n",
    "    if e.response[\"Error\"][\"Code\"] == \"EntityAlreadyExists\":\n",
    "        iam_role_kinesis = iam.get_role(RoleName=iam_kinesis_role_name)\n",
    "        print(\"Role already exists. That is OK.\")\n",
    "        iam_kinesis_role_passed = True\n",
    "    else:\n",
    "        print(\"Unexpected error: %s\" % e)\n",
    "\n",
    "time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_role_kinesis_name = iam_role_kinesis[\"Role\"][\"RoleName\"]\n",
    "print(\"Role Name: {}\".format(iam_role_kinesis_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_role_kinesis_arn = iam_role_kinesis[\"Role\"][\"Arn\"]\n",
    "print(\"Role ARN: {}\".format(iam_role_kinesis_arn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "account_id = sts.get_caller_identity()[\"Account\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specify Stream Name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stream_name = \"dsoaws-kinesis-data-stream\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specify Firehose Name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "firehose_name = \"dsoaws-kinesis-data-firehose\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specify Lambda Function Name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_fn_name_cloudwatch = \"DeliverKinesisAnalyticsToCloudWatch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_fn_name_invoke_sm_endpoint = \"InvokeSageMakerEndpointFromKinesis\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_fn_name_sns = \"PushNotificationToSNS\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "kinesis_policy_doc = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"s3:AbortMultipartUpload\",\n",
    "                \"s3:GetBucketLocation\",\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\",\n",
    "                \"s3:ListBucketMultipartUploads\",\n",
    "                \"s3:PutObject\",\n",
    "            ],\n",
    "            \"Resource\": [\"arn:aws:s3:::{}\".format(bucket), \"arn:aws:s3:::{}/*\".format(bucket)],\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\"logs:PutLogEvents\"],\n",
    "            \"Resource\": [\"arn:aws:logs:{}:{}:log-group:/*\".format(region, account_id)],\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"kinesis:*\",\n",
    "            ],\n",
    "            \"Resource\": [\"arn:aws:kinesis:{}:{}:stream/{}\".format(region, account_id, stream_name)],\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"firehose:*\",\n",
    "            ],\n",
    "            \"Resource\": [\"arn:aws:firehose:{}:{}:deliverystream/{}\".format(region, account_id, firehose_name)],\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"kinesisanalytics:*\",\n",
    "            ],\n",
    "            \"Resource\": [\"*\"],\n",
    "        },\n",
    "        {\n",
    "            \"Sid\": \"UseLambdaFunction\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\"lambda:InvokeFunction\", \"lambda:GetFunctionConfiguration\"],\n",
    "            \"Resource\": [\"*\"],\n",
    "        },\n",
    "        {\"Effect\": \"Allow\", \"Action\": \"iam:PassRole\", \"Resource\": [\"arn:aws:iam::*:role/service-role/kinesis*\"]},\n",
    "    ],\n",
    "}\n",
    "\n",
    "print(json.dumps(kinesis_policy_doc, indent=4, sort_keys=True, default=str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Update Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "response = iam.put_role_policy(\n",
    "    RoleName=iam_role_kinesis_name, PolicyName=\"DSOAWS_KinesisPolicy\", PolicyDocument=json.dumps(kinesis_policy_doc)\n",
    ")\n",
    "\n",
    "time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(json.dumps(response, indent=4, sort_keys=True, default=str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create AWS Lambda IAM Role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_lambda_role_name = \"DSOAWS_Lambda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_lambda_role_passed = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assume_role_policy_doc = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\"Effect\": \"Allow\", \"Principal\": {\"Service\": \"lambda.amazonaws.com\"}, \"Action\": \"sts:AssumeRole\"},\n",
    "        {\"Effect\": \"Allow\", \"Principal\": {\"Service\": \"kinesisanalytics.amazonaws.com\"}, \"Action\": \"sts:AssumeRole\"},\n",
    "    ],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "try:\n",
    "    iam_role_lambda = iam.create_role(\n",
    "        RoleName=iam_lambda_role_name,\n",
    "        AssumeRolePolicyDocument=json.dumps(assume_role_policy_doc),\n",
    "        Description=\"DSOAWS Lambda Role\",\n",
    "    )\n",
    "    print(\"Role succesfully created.\")\n",
    "    iam_lambda_role_passed = True\n",
    "except ClientError as e:\n",
    "    if e.response[\"Error\"][\"Code\"] == \"EntityAlreadyExists\":\n",
    "        iam_role_lambda = iam.get_role(RoleName=iam_lambda_role_name)\n",
    "        print(\"Role already exists. This is OK.\")\n",
    "        iam_lambda_role_passed = True\n",
    "    else:\n",
    "        print(\"Unexpected error: %s\" % e)\n",
    "\n",
    "time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_role_lambda_name = iam_role_lambda[\"Role\"][\"RoleName\"]\n",
    "print(\"Role Name: {}\".format(iam_role_lambda_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam_role_lambda_arn = iam_role_lambda[\"Role\"][\"Arn\"]\n",
    "print(\"Role ARN: {}\".format(iam_role_lambda_arn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create AWS Lambda IAM Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_policy_doc = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"UseLambdaFunction\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\"lambda:InvokeFunction\", \"lambda:GetFunctionConfiguration\"],\n",
    "            \"Resource\": \"arn:aws:lambda:{}:{}:function:*\".format(region, account_id),\n",
    "        },\n",
    "        {\"Effect\": \"Allow\", \"Action\": \"cloudwatch:*\", \"Resource\": \"*\"},\n",
    "        {\"Effect\": \"Allow\", \"Action\": \"sns:*\", \"Resource\": \"*\"},\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": \"logs:CreateLogGroup\",\n",
    "            \"Resource\": \"arn:aws:logs:{}:{}:*\".format(region, account_id),\n",
    "        },\n",
    "        {\"Effect\": \"Allow\", \"Action\": \"sagemaker:InvokeEndpoint\", \"Resource\": \"*\"},\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\"logs:CreateLogStream\", \"logs:PutLogEvents\"],\n",
    "            \"Resource\": \"arn:aws:logs:{}:{}:log-group:/aws/lambda/*\".format(region, account_id),\n",
    "        },\n",
    "    ],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(json.dumps(lambda_policy_doc, indent=4, sort_keys=True, default=str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "response = iam.put_role_policy(\n",
    "    RoleName=iam_role_lambda_name, PolicyName=\"DSOAWS_LambdaPolicy\", PolicyDocument=json.dumps(lambda_policy_doc)\n",
    ")\n",
    "\n",
    "time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(json.dumps(response, indent=4, sort_keys=True, default=str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store Variables for Next Notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store stream_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store firehose_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_kinesis_role_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_role_kinesis_arn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_lambda_role_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_role_lambda_arn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store lambda_fn_name_cloudwatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store lambda_fn_name_invoke_sm_endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store lambda_fn_name_sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_kinesis_role_passed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store iam_lambda_role_passed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Release Resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "\n",
    "<p><b>Shutting down your kernel for this notebook to release resources.</b></p>\n",
    "<button class=\"sm-command-button\" data-commandlinker-command=\"kernelmenu:shutdown\" style=\"display:none;\">Shutdown Kernel</button>\n",
    "        \n",
    "<script>\n",
    "try {\n",
    "    els = document.getElementsByClassName(\"sm-command-button\");\n",
    "    els[0].click();\n",
    "}\n",
    "catch(err) {\n",
    "    // NoOp\n",
    "}    \n",
    "</script>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "\n",
    "try {\n",
    "    Jupyter.notebook.save_checkpoint();\n",
    "    Jupyter.notebook.session.delete();\n",
    "}\n",
    "catch(err) {\n",
    "    // NoOp\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
